import apachelog
import sys
import re

from urldb.control.populate import Populate

''' 
'''
class LogParser(object):

    populate = None

    def __init__(self, options):
        format = apachelog.formats[options.format];
        lp = apachelog.parser(format);
        self.populate = Populate(options)
        self.populate.set_ignore_list(options.ignore)

        log = options.log
        try:
            for line in open(log):
                try:
                    data = lp.parse(line)
                    url = data['%r']
                    if options.name is not None:
                        url = re.sub('^GET ', options.name, url);
                    else:
                        url = re.sub('^GET ', '', url);
                    url = re.sub(' HTTP/[1-9]+.[0-9]+', '', url);
                    # if filtering based on a -q string, then look here and continue (rather than persist the url) if not found
                    if options.query != None and url.find(options.query) < 0:
                        continue;
                    self.populate.new_url(url)
                except:
                   sys.stderr.write("Unable to parse %s" % line)
        except:
            sys.stderr.write("Couldn't open file %s \n" % log)
